package com.crawler.util.html;

import com.crawler.util.StrKit;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.List;

/**
 * @author guorui1
 */
public class LinkExtractor {

    public static List<SourceLink> extract(String url, String html) {
        List<SourceLink> links = new ArrayList<>();
        Document document = cleanHtml(url, html);
        Elements elements = document.select("a");
        for (Element elem : elements) {
            String href = elem.absUrl("href");
            String text = elem.text();
            if (StrKit.isEmpty(href)) {
                continue;
            }
            href = StrKit.cleanUrl(href);
            if (isTimeLink(text) || isShortTextLink(text)) {
                continue;
            }
            links.add(new SourceLink(href, text));
        }
        return links;
    }

    public static Document cleanHtml(String url, String html) {
        Document dirtyDocument = Jsoup.parse(html, url);
        Cleaner cleaner = new Cleaner(Whitelist.basic());
        Document document = cleaner.clean(dirtyDocument);
        document.select("script,style,option,link,img,textarea,input,object").remove();
        document.select(":empty").remove();
        return document;
    }

    public static boolean isTimeLink(String text) {
        return false;
    }

    public static boolean isShortTextLink(String text) {
        return StrKit.isEmpty(text) || text.length() <= 4;
    }
}
